{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-11-19T09:11:46.698277100Z",
     "start_time": "2024-11-19T09:11:46.690756900Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "m1=tensor([[7, 7, 1, 1, 9, 4],\n",
      "        [1, 5, 9, 7, 5, 7],\n",
      "        [6, 3, 4, 3, 7, 5],\n",
      "        [8, 5, 3, 6, 2, 9]], dtype=torch.int32)\n",
      "m2=tensor([[1, 6, 6, 8],\n",
      "        [5, 8, 5, 1],\n",
      "        [6, 8, 2, 8],\n",
      "        [3, 6, 4, 7],\n",
      "        [7, 3, 5, 8],\n",
      "        [3, 8, 3, 3]], dtype=torch.int32)\n",
      "m3=m1*m2=tensor([[126, 171, 140, 162],\n",
      "        [157, 231, 123, 195],\n",
      "        [118, 171, 121, 175],\n",
      "        [110, 226, 140, 178]], dtype=torch.int32)\n",
      "\n",
      "左纵：\n",
      "m1_0=tensor([[7, 7, 1],\n",
      "        [1, 5, 9],\n",
      "        [6, 3, 4],\n",
      "        [8, 5, 3]], dtype=torch.int32)\n",
      "m1_1=tensor([[1, 9, 4],\n",
      "        [7, 5, 7],\n",
      "        [3, 7, 5],\n",
      "        [6, 2, 9]], dtype=torch.int32)\n",
      "\n",
      "右横：\n",
      "m2_0=tensor([[1, 6, 6, 8],\n",
      "        [5, 8, 5, 1],\n",
      "        [6, 8, 2, 8]], dtype=torch.int32)\n",
      "m2_1=tensor([[3, 6, 4, 7],\n",
      "        [7, 3, 5, 8],\n",
      "        [3, 8, 3, 3]], dtype=torch.int32)\n",
      "左纵右横矩阵乘，然后相加的结果=tensor([[126, 171, 140, 162],\n",
      "        [157, 231, 123, 195],\n",
      "        [118, 171, 121, 175],\n",
      "        [110, 226, 140, 178]], dtype=torch.int32)\n",
      "直接matmul和切分后matmul的结果是否相等：True\n"
     ]
    }
   ],
   "source": [
    "# 矩阵的横切和纵切\n",
    "import torch\n",
    "\n",
    "m1 = torch.randint(1, 10, [4,6], dtype=torch.int32)\n",
    "print(f'm1={m1}')\n",
    "\n",
    "m2 = torch.randint(1, 10, [4,6], dtype=torch.int32).transpose(-1, -2)\n",
    "print(f'm2={m2}')\n",
    "\n",
    "m3 = torch.matmul(m1, m2)  # [4,6] * [6,4]\n",
    "print(f'm3=m1*m2={m3}')\n",
    "\n",
    "# 纵切 * 横切\n",
    "# 左纵右横，结果相加\n",
    "\n",
    "# 左纵  \n",
    "# [4,3] | [4,3]\n",
    "m1_0, m1_1 = m1[:,0:3],m1[:,3:6]\n",
    "print(f'\\n左纵：')\n",
    "print(f'm1_0={m1_0}')\n",
    "print(f'm1_1={m1_1}')\n",
    "\n",
    "# 右横\n",
    "# [3,4]\n",
    "# -----\n",
    "# [3,4]\n",
    "m2_0, m2_1 = m2[0:3,:], m2[3:6,:]\n",
    "print(f'\\n右横：')\n",
    "print(f'm2_0={m2_0}')\n",
    "print(f'm2_1={m2_1}')\n",
    "\n",
    "m4 = torch.add(torch.matmul(m1_0, m2_0), torch.matmul(m1_1, m2_1))\n",
    "print(f'左纵右横矩阵乘，然后相加的结果={m4}')\n",
    "\n",
    "print(f'直接matmul和切分后matmul的结果是否相等：{torch.equal(m4, m3)}')\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "# flash attention原理简介：https://zhuanlan.zhihu.com/p/657004029\n"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "7f138e4fc6976605"
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "kv_block_rows=3\n",
      "q_block_rows=2\n",
      "外层循环(KV)-0\n",
      "6\n",
      "k1.shape=torch.Size([4, 3])\n",
      "v1.shape=torch.Size([3, 4])\n",
      "---内层循环(Q)-0\n",
      "---内层循环(Q)-1\n",
      "---内层循环(Q)-2\n",
      "外层循环(KV)-1\n",
      "6\n",
      "k1.shape=torch.Size([4, 3])\n",
      "v1.shape=torch.Size([3, 4])\n",
      "---内层循环(Q)-0\n",
      "---内层循环(Q)-1\n",
      "---内层循环(Q)-2\n",
      "通过循环计算o=tensor([[2528., 2047., 2046., 2301.],\n",
      "        [2113., 1853., 1748., 2312.],\n",
      "        [1869., 1515., 1313., 1976.],\n",
      "        [3168., 2736., 2700., 3228.],\n",
      "        [2380., 2009., 2143., 2125.],\n",
      "        [1836., 1401., 1316., 1639.]])\n",
      "直接矩阵乘计算o2=tensor([[2528, 2047, 2046, 2301],\n",
      "        [2113, 1853, 1748, 2312],\n",
      "        [1869, 1515, 1313, 1976],\n",
      "        [3168, 2736, 2700, 3228],\n",
      "        [2380, 2009, 2143, 2125],\n",
      "        [1836, 1401, 1316, 1639]])\n",
      "o==o2 ? True\n"
     ]
    }
   ],
   "source": [
    "# flash attention原理：https://www.bilibili.com/video/BV1UT421k7rA/?spm_id_from=333.337.search-card.all.click&vd_source=0fc0894df0a5027926533881a589f377\n",
    "# 如下代码，实验没有softmax的QKV切块相乘、及循环更新输出o\n",
    "import torch\n",
    "\n",
    "q = torch.randint(0, 10, [6, 4])\n",
    "k = torch.randint(0, 10, [6, 4]).transpose(-1, -2).contiguous()\n",
    "v = torch.randint(0, 10, [6, 4])\n",
    "o = torch.zeros([6, 4])\n",
    "\n",
    "outer_times = 2\n",
    "inner_times = 3\n",
    "\n",
    "kv_block_rows = k.shape[1] // outer_times  # 外层循环每次处理k的列数量、也即v的行数量，二者必须相等\n",
    "q_block_rows = q.shape[0] // inner_times  # 内层循环每次处理q的行数量，与kv每次处理的数量可以不等，可随便取\n",
    "print(f'kv_block_rows={kv_block_rows}')\n",
    "print(f'q_block_rows={q_block_rows}')\n",
    "\n",
    "for i in range(outer_times):\n",
    "    print(f'外层循环(KV)-{i}')\n",
    "    print(k.shape[1])\n",
    "    \n",
    "    k1 = k[:, i * kv_block_rows : (i + 1) * kv_block_rows]\n",
    "    print(f'k1.shape={k1.shape}')\n",
    "    \n",
    "    v1 = v[i * kv_block_rows : (i + 1) * kv_block_rows, :]\n",
    "    print(f'v1.shape={v1.shape}')\n",
    "    \n",
    "    for j in range(inner_times):\n",
    "        print(f'---内层循环(Q)-{j}')\n",
    "        q1 = q[j * q_block_rows : (j + 1) * q_block_rows, :]\n",
    "        # print(f'---q1.shape={q1.shape}')\n",
    "        score1 = torch.matmul(q1, k1)\n",
    "        # print(f'---score1.shape={score1.shape}')\n",
    "        o1 = torch.matmul(score1, v1)\n",
    "        # print(f'---o1.shape={o1.shape}')\n",
    "        o[j * q_block_rows : (j + 1) * q_block_rows, :] += o1\n",
    "    \n",
    "    \n",
    "print(f'通过循环计算o={o}')\n",
    "\n",
    "o2 = torch.matmul(torch.matmul(q, k), v)\n",
    "print(f'直接矩阵乘计算o2={o2}')\n",
    "\n",
    "print(f'o==o2 ? {torch.equal(o, o2)}')\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-11-28T03:20:36.489680100Z",
     "start_time": "2024-11-28T03:20:36.476116800Z"
    }
   },
   "id": "3e7260ac8ef6e807"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "q = torch.randint(0, 10, [6, 4])\n",
    "k = torch.randint(0, 10, [6, 4]).transpose(-1, -2).contiguous()\n",
    "v = torch.randint(0, 10, [6, 4])\n",
    "o = torch.zeros([6, 4])\n",
    "\n",
    "print(q)\n",
    "print(o)\n",
    "for i in range(3):\n",
    "    o[i * 2: (i+1) * 2] = q[i * 2: (i+1) * 2]\n",
    "    print(o)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "6faf02d81eda6433"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import torch\n",
    "\n",
    "q = torch.randint(0, 10, [6, 4])\n",
    "k = torch.randint(0, 10, [6, 4]).transpose(-1, -2).contiguous()\n",
    "v = torch.randint(0, 10, [6, 4])\n",
    "o = torch.ones([6, 4])\n",
    "\n",
    "print(q)\n",
    "print(o)\n",
    "for i in range(3):\n",
    "    o[i * 2: (i+1) * 2] += q[i * 2: (i+1) * 2]\n",
    "    print(o)"
   ],
   "metadata": {
    "collapsed": false
   },
   "id": "b2adca8865cc1502"
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------Attention : Q0 * K0 * V0---------\n",
      "O0.shape=torch.Size([1, 1, 3, 4])\n",
      "O0=tensor([[[[-0.8009,  1.0935,  0.9991, -0.0539],\n",
      "          [ 0.7020,  0.3334,  0.2951,  0.4763],\n",
      "          [ 0.3863,  0.6345,  0.4237, -0.2622]]]], grad_fn=<AddBackward0>)\n",
      "O1=tensor([[[[0., 0., 0., 0.],\n",
      "          [0., 0., 0., 0.],\n",
      "          [0., 0., 0., 0.]]]], grad_fn=<SplitBackward0>)\n",
      "\n",
      "\n",
      "-----------Attention : Q1 * K0 * V0---------\n",
      "O1.shape=torch.Size([1, 1, 3, 4])\n",
      "O0=tensor([[[[-0.8009,  1.0935,  0.9991, -0.0539],\n",
      "          [ 0.7020,  0.3334,  0.2951,  0.4763],\n",
      "          [ 0.3863,  0.6345,  0.4237, -0.2622]]]], grad_fn=<AddBackward0>)\n",
      "O1=tensor([[[[-0.0762,  1.0155,  0.6203, -1.0781],\n",
      "          [ 0.2450,  0.5358,  0.5131,  0.4425],\n",
      "          [-0.6420,  0.9913,  0.9276,  0.0987]]]], grad_fn=<AddBackward0>)\n",
      "\n",
      "\n",
      "-----------Attention : Q0 * K1 * V1---------\n",
      "O0.shape=torch.Size([1, 1, 3, 4])\n",
      "O0=tensor([[[[-0.5268,  0.8991,  0.5556,  0.0493],\n",
      "          [ 0.6792,  0.3549,  0.1924,  0.4776],\n",
      "          [ 0.3602,  0.6313, -0.0579, -0.0510]]]], grad_fn=<AddBackward0>)\n",
      "O1=tensor([[[[-0.0762,  1.0155,  0.6203, -1.0781],\n",
      "          [ 0.2450,  0.5358,  0.5131,  0.4425],\n",
      "          [-0.6420,  0.9913,  0.9276,  0.0987]]]], grad_fn=<AddBackward0>)\n",
      "\n",
      "\n",
      "-----------Attention : Q1 * K1 * V1---------\n",
      "O1.shape=torch.Size([1, 1, 3, 4])\n",
      "O0=tensor([[[[-0.5268,  0.8991,  0.5556,  0.0493],\n",
      "          [ 0.6792,  0.3549,  0.1924,  0.4776],\n",
      "          [ 0.3602,  0.6313, -0.0579, -0.0510]]]], grad_fn=<AddBackward0>)\n",
      "O1=tensor([[[[ 0.1526,  0.6189, -0.0653, -0.3444],\n",
      "          [ 0.2587,  0.5445,  0.3678,  0.4298],\n",
      "          [-0.4667,  0.7881,  0.1669,  0.4734]]]], grad_fn=<AddBackward0>)\n",
      "\n",
      "\n",
      "O.shape=torch.Size([1, 1, 6, 4])\n",
      "O=tensor([[[[-0.5268,  0.8991,  0.5556,  0.0493],\n",
      "          [ 0.6792,  0.3549,  0.1924,  0.4776],\n",
      "          [ 0.3602,  0.6313, -0.0579, -0.0510],\n",
      "          [ 0.1526,  0.6189, -0.0653, -0.3444],\n",
      "          [ 0.2587,  0.5445,  0.3678,  0.4298],\n",
      "          [-0.4667,  0.7881,  0.1669,  0.4734]]]], grad_fn=<CatBackward0>)\n",
      "(rowsum) l.shape=torch.Size([1, 1, 6, 1])\n",
      "(rowsum) l=tensor([[[[1.8269],\n",
      "          [1.0756],\n",
      "          [2.1055],\n",
      "          [3.5538],\n",
      "          [1.4726],\n",
      "          [2.3199]]]], grad_fn=<CatBackward0>)\n",
      "(rowmax) m.shape=torch.Size([1, 1, 6, 1])\n",
      "(rowmax) m=tensor([[[[1.8104],\n",
      "          [4.5761],\n",
      "          [1.8774],\n",
      "          [0.2747],\n",
      "          [3.6494],\n",
      "          [1.6429]]]], grad_fn=<CatBackward0>)\n",
      "O=tensor([[[[-0.5268,  0.8991,  0.5556,  0.0493],\n",
      "          [ 0.6792,  0.3549,  0.1924,  0.4776],\n",
      "          [ 0.3602,  0.6313, -0.0579, -0.0510],\n",
      "          [ 0.1526,  0.6189, -0.0653, -0.3444],\n",
      "          [ 0.2587,  0.5445,  0.3678,  0.4298],\n",
      "          [-0.4667,  0.7881,  0.1669,  0.4734]]]], grad_fn=<CatBackward0>)\n",
      "O_=tensor([[[[-0.5268,  0.8991,  0.5556,  0.0493],\n",
      "          [ 0.6792,  0.3549,  0.1924,  0.4776],\n",
      "          [ 0.3602,  0.6313, -0.0579, -0.0510],\n",
      "          [ 0.1526,  0.6189, -0.0653, -0.3444],\n",
      "          [ 0.2587,  0.5445,  0.3678,  0.4298],\n",
      "          [-0.4667,  0.7881,  0.1669,  0.4734]]]],\n",
      "       grad_fn=<UnsafeViewBackward0>)\n",
      "O - O_ ? tensor([[[[ 0.0000e+00, -5.9605e-08,  0.0000e+00, -1.4901e-08],\n",
      "          [-5.9605e-08,  5.9605e-08,  1.4901e-08,  0.0000e+00],\n",
      "          [ 0.0000e+00, -5.9605e-08,  1.4901e-08,  7.4506e-09],\n",
      "          [ 0.0000e+00, -5.9605e-08, -2.9802e-08,  5.9605e-08],\n",
      "          [ 5.9605e-08,  5.9605e-08,  0.0000e+00,  0.0000e+00],\n",
      "          [ 1.1921e-07, -1.1921e-07,  0.0000e+00, -8.9407e-08]]]],\n",
      "       grad_fn=<SubBackward0>)\n",
      "O == O_ ? tensor([[[[True, True, True, True],\n",
      "          [True, True, True, True],\n",
      "          [True, True, True, True],\n",
      "          [True, True, True, True],\n",
      "          [True, True, True, True],\n",
      "          [True, True, True, True]]]])\n"
     ]
    }
   ],
   "source": [
    "# flash attention 论文代码实现\n",
    "# 参考：https://zhuanlan.zhihu.com/p/676655352\n",
    "\n",
    "# 前向过程步骤详解：\n",
    "# 1-5：主要在初始化和进行切分\n",
    "# 6-7：遍历K，V的每一块（Outer Loop）\n",
    "# 8：遍历Q的每一块 (Inner Loop)\n",
    "# 9：将分块后的QKV的小块加载到SRAM (Copy Block to SRAM)\n",
    "# 10：计算Sij (Compute Block on SRAM)\n",
    "# 11：计算Sij mask (Compute Block on SRAM)\n",
    "# 12：计算m,l统计量 (Compute Block on SRAM)\n",
    "# 13：计算m,l统计量 (Compute Block on SRAM)\n",
    "# 14：dropout (Compute Block on SRAM)\n",
    "# 15：计算Oi并写入HBM (Output to HBM)\n",
    "# 16：把li,mi写入HBM (Output to HBM)\n",
    "\n",
    "import torch\n",
    "\n",
    "NEG_INF = -1e10  # -infinity\n",
    "EPSILON = 1e-10\n",
    "\n",
    "# Q矩阵：[Q_LEN,d]\n",
    "# K矩阵：[K_KEN,d]\n",
    "# Q * K^T = S  [Q_LEN,K_LEN]\n",
    "# V矩阵：[K_LEN,d]\n",
    "# 即：K矩阵和V矩阵的shape必须相同\n",
    "Q_LEN = 6\n",
    "K_LEN = 6\n",
    "Q_BLOCK_SIZE = 3\n",
    "KV_BLOCK_SIZE = 3\n",
    "P_DROP = 0.2\n",
    "\n",
    "Tr = Q_LEN // Q_BLOCK_SIZE   # Q分块数\n",
    "Tc = K_LEN // KV_BLOCK_SIZE  # K/V分块数\n",
    "\n",
    "Q = torch.randn(1, 1, Q_LEN, 4, requires_grad=True).to(device='cpu')\n",
    "K = torch.randn(1, 1, K_LEN, 4, requires_grad=True).to(device='cpu')\n",
    "V = torch.randn(1, 1, K_LEN, 4, requires_grad=True).to(device='cpu')\n",
    "\n",
    "O = torch.zeros_like(Q, requires_grad=True)\n",
    "l = torch.zeros(Q.shape[:-1])[..., None]\n",
    "m = torch.ones(Q.shape[:-1])[..., None] * NEG_INF\n",
    "\n",
    "# step 4\n",
    "Q_BLOCKS = torch.split(Q, Q_BLOCK_SIZE, dim=2)  # split之后得到一个tuple\n",
    "K_BLOCKS = torch.split(K, KV_BLOCK_SIZE, dim=2)\n",
    "V_BLOCKS = torch.split(V, KV_BLOCK_SIZE, dim=2)\n",
    "# print(f'Q_BLOCKS={Q_BLOCKS}')\n",
    "# print(f'Q.shape={Q.shape}')\n",
    "\n",
    "# step 5\n",
    "O_BLOCKS = list(torch.split(O, Q_BLOCK_SIZE, dim=2))\n",
    "l_BLOCKS = list(torch.split(l, Q_BLOCK_SIZE, dim=2))\n",
    "m_BLOCKS = list(torch.split(m, Q_BLOCK_SIZE, dim=2))\n",
    "\n",
    "# step 6\n",
    "for j in range(Tc):\n",
    "    # step 7\n",
    "    Kj = K_BLOCKS[j]\n",
    "    Vj = V_BLOCKS[j]\n",
    "    # step 8\n",
    "    for i in range(Tr):\n",
    "        # step 9\n",
    "        Qi = Q_BLOCKS[i]\n",
    "        Oi = O_BLOCKS[i]\n",
    "        li = l_BLOCKS[i]\n",
    "        mi = m_BLOCKS[i]\n",
    "\n",
    "        # step 10  使用爱<因斯坦求和约定>方式进行 矩阵乘\n",
    "        S_ij = torch.einsum('... i d, ... j d -> ... i j', Qi, Kj)  # [Br,Bc]\n",
    "\n",
    "        # step 11\n",
    "        # mask = S_ij.ge(0.5)\n",
    "        # S_ij = torch.masked_fill(S_ij, mask, value=0)\n",
    "        \n",
    "        # step 12\n",
    "        m_block_ij, _ = torch.max(S_ij, dim=-1, keepdims=True)  # rowmax，即按行取最大值，[Br,1]\n",
    "        P_ij = torch.exp(S_ij - m_block_ij)  # [Br,Bc] - [Br,1]，其中[Br,1]先广播为[Br,Bc]，然后做elewise减法\n",
    "        l_block_ij = torch.sum(P_ij, dim=-1, keepdims=True) + EPSILON  # rowsum，按行求和，[Br,1]\n",
    "        P_ij_Vj = torch.einsum('... i j, ... j d -> ... i d', P_ij, Vj)  # P_ij和V_j做矩阵乘\n",
    "\n",
    "        # step 13\n",
    "        mi_new = torch.maximum(m_block_ij, mi)  # 更新rowmax，即mi，两个矩阵对应位置的元素取较大值\n",
    "\n",
    "        li_new = torch.exp(mi - mi_new) * li + \\\n",
    "                 torch.exp(m_block_ij - mi_new) * l_block_ij  # 更新rowsum，即li\n",
    "\n",
    "        # step 14\n",
    "        # m = torch.nn.Dropout(p=P_DROP)\n",
    "        # P_ij_Vj = m(P_ij_Vj)  # 随机丢弃元素，防止过拟合\n",
    "\n",
    "        # Step 15\n",
    "        O_BLOCKS[i] = (li / li_new) * torch.exp(mi - mi_new) * Oi \\\n",
    "                      + (torch.exp(m_block_ij - mi_new) / li_new) * P_ij_Vj\n",
    "        print(f'-----------Attention : Q{i} * K{j} * V{j}---------')\n",
    "        print(f'O{i}.shape={O_BLOCKS[i].shape}')\n",
    "        # 打印O的每个block\n",
    "        for k in range(Tr):\n",
    "            print(f'O{k}={O_BLOCKS[k]}')\n",
    "        print('\\n')\n",
    "\n",
    "        # step 16\n",
    "        l_BLOCKS[i] = li_new\n",
    "        m_BLOCKS[i] = mi_new\n",
    "\n",
    "O = torch.cat(O_BLOCKS, dim=2)\n",
    "l = torch.cat(l_BLOCKS, dim=2)\n",
    "m = torch.cat(m_BLOCKS, dim=2)\n",
    "\n",
    "print(f'O.shape={O.shape}')\n",
    "print(f'O={O}')\n",
    "print(f'(rowsum) l.shape={l.shape}')\n",
    "print(f'(rowsum) l={l}')\n",
    "print(f'(rowmax) m.shape={m.shape}')\n",
    "print(f'(rowmax) m={m}')\n",
    "\n",
    "\n",
    "### 如上是flash attention的实现，下面使用经典attention实现\n",
    "Kt = K.transpose(-1, -2)\n",
    "Q_K = torch.matmul(Q, Kt)\n",
    "softmax_ = torch.softmax(Q_K, dim=-1)\n",
    "O_ = torch.matmul(softmax_, V)\n",
    "print(f'O={O}')\n",
    "print(f'O_={O_}')\n",
    "print(f'O - O_ ? {O - O_}')\n",
    "print(f'O == O_ ? {(O - O_) < 1e-6}')\n",
    "\n",
    "\n",
    "\n"
   ],
   "metadata": {
    "collapsed": false,
    "ExecuteTime": {
     "end_time": "2024-11-30T10:01:02.979713700Z",
     "start_time": "2024-11-30T10:01:02.943588600Z"
    }
   },
   "id": "43a609023d006abe"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "6a264611066703d9"
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false
   },
   "id": "74f51ba4cc6be3ba"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
